{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import os\n",
    "import tushare as ts\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import sys\n",
    "from IPython.display import clear_output\n",
    "\n",
    "sys.path.append(\"./utils/\")\n",
    "import date_util\n",
    "import token_util\n",
    "import data_util\n",
    "pro = token_util.set_token()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 拉取股票列表 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 默认一年前,默认市值小于1000亿,将时间设置在交易日,默认剔除滚动市盈率为负的股票\n",
    "stock_list = data_util.get_stock_list(trade_date='20200429')\n",
    "\n",
    "# stock_list = pd.read_csv(\"./data_pulled/stock_date_delta_priceNone.csv\")\n",
    "# stock_list.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "root_dir = \"data_pulled\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 拉取或更新日k数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(stock_list)):\n",
    "    \n",
    "    try:\n",
    "        \n",
    "        stock_code = stock_list.iloc[i][\"ts_code\"]\n",
    "        data= data_util.pull_data(stock_code, root_dir=root_dir)\n",
    "        \n",
    "    except:\n",
    "        time.sleep(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 拉取或更新周k数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(stock_list)):\n",
    "        \n",
    "    try:\n",
    "\n",
    "        stock_code = stock_list.iloc[i][\"ts_code\"]\n",
    "        data= data_util.pull_data(stock_code, freq='W', root_dir=root_dir)\n",
    "\n",
    "    except:\n",
    "        \n",
    "        time.sleep(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 拉取或更新月k数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HTTPConnectionPool(host='api.waditu.com', port=80): Read timed out.\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "HTTPConnectionPool(host='api.waditu.com', port=80): Read timed out.\n",
      "HTTPConnectionPool(host='api.waditu.com', port=80): Read timed out.\n",
      "HTTPConnectionPool(host='api.waditu.com', port=80): Read timed out.\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "抱歉，您每分钟最多访问该接口120次，权限的具体详情访问：https://tushare.pro/document/1?doc_id=108。\n",
      "HTTPConnectionPool(host='api.waditu.com', port=80): Read timed out. (read timeout=15)\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(stock_list)):\n",
    "        \n",
    "    try:\n",
    "        \n",
    "        stock_code = stock_list.iloc[i][\"ts_code\"]\n",
    "        data= data_util.pull_data(stock_code, freq='M', root_dir=root_dir)\n",
    "        \n",
    "    except:\n",
    "        \n",
    "        time.sleep(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ts_code</th>\n",
       "      <th>trade_date</th>\n",
       "      <th>close</th>\n",
       "      <th>open</th>\n",
       "      <th>high</th>\n",
       "      <th>low</th>\n",
       "      <th>vol</th>\n",
       "      <th>amount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>000001.SZ</td>\n",
       "      <td>20191129</td>\n",
       "      <td>15.29</td>\n",
       "      <td>16.35</td>\n",
       "      <td>17.44</td>\n",
       "      <td>15.18</td>\n",
       "      <td>1.640143e+09</td>\n",
       "      <td>2.672014e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000001.SZ</td>\n",
       "      <td>20191031</td>\n",
       "      <td>16.26</td>\n",
       "      <td>15.60</td>\n",
       "      <td>17.60</td>\n",
       "      <td>15.60</td>\n",
       "      <td>2.019234e+09</td>\n",
       "      <td>3.355394e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000001.SZ</td>\n",
       "      <td>20190930</td>\n",
       "      <td>15.59</td>\n",
       "      <td>14.15</td>\n",
       "      <td>16.04</td>\n",
       "      <td>14.11</td>\n",
       "      <td>2.538655e+09</td>\n",
       "      <td>3.799487e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>000001.SZ</td>\n",
       "      <td>20190830</td>\n",
       "      <td>14.16</td>\n",
       "      <td>14.06</td>\n",
       "      <td>15.22</td>\n",
       "      <td>13.03</td>\n",
       "      <td>2.993290e+09</td>\n",
       "      <td>4.331356e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>000001.SZ</td>\n",
       "      <td>20190731</td>\n",
       "      <td>14.13</td>\n",
       "      <td>14.01</td>\n",
       "      <td>14.55</td>\n",
       "      <td>13.44</td>\n",
       "      <td>1.652336e+09</td>\n",
       "      <td>2.307291e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>000001.SZ</td>\n",
       "      <td>20190628</td>\n",
       "      <td>13.78</td>\n",
       "      <td>12.22</td>\n",
       "      <td>13.95</td>\n",
       "      <td>11.60</td>\n",
       "      <td>1.662158e+09</td>\n",
       "      <td>2.137764e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>000001.SZ</td>\n",
       "      <td>20190531</td>\n",
       "      <td>12.18</td>\n",
       "      <td>13.10</td>\n",
       "      <td>13.35</td>\n",
       "      <td>11.93</td>\n",
       "      <td>1.864961e+09</td>\n",
       "      <td>2.337029e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>000001.SZ</td>\n",
       "      <td>20190430</td>\n",
       "      <td>13.85</td>\n",
       "      <td>12.83</td>\n",
       "      <td>14.84</td>\n",
       "      <td>12.83</td>\n",
       "      <td>2.593170e+09</td>\n",
       "      <td>3.620546e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>000001.SZ</td>\n",
       "      <td>20190329</td>\n",
       "      <td>12.82</td>\n",
       "      <td>12.48</td>\n",
       "      <td>13.38</td>\n",
       "      <td>12.01</td>\n",
       "      <td>2.671303e+09</td>\n",
       "      <td>3.364583e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>000001.SZ</td>\n",
       "      <td>20190228</td>\n",
       "      <td>12.36</td>\n",
       "      <td>11.20</td>\n",
       "      <td>12.66</td>\n",
       "      <td>10.90</td>\n",
       "      <td>1.911474e+09</td>\n",
       "      <td>2.230669e+10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>000001.SZ</td>\n",
       "      <td>20190131</td>\n",
       "      <td>11.10</td>\n",
       "      <td>9.39</td>\n",
       "      <td>11.20</td>\n",
       "      <td>9.15</td>\n",
       "      <td>1.816315e+09</td>\n",
       "      <td>1.876003e+10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ts_code trade_date  close   open   high    low           vol  \\\n",
       "0   000001.SZ   20191129  15.29  16.35  17.44  15.18  1.640143e+09   \n",
       "1   000001.SZ   20191031  16.26  15.60  17.60  15.60  2.019234e+09   \n",
       "2   000001.SZ   20190930  15.59  14.15  16.04  14.11  2.538655e+09   \n",
       "3   000001.SZ   20190830  14.16  14.06  15.22  13.03  2.993290e+09   \n",
       "4   000001.SZ   20190731  14.13  14.01  14.55  13.44  1.652336e+09   \n",
       "5   000001.SZ   20190628  13.78  12.22  13.95  11.60  1.662158e+09   \n",
       "6   000001.SZ   20190531  12.18  13.10  13.35  11.93  1.864961e+09   \n",
       "7   000001.SZ   20190430  13.85  12.83  14.84  12.83  2.593170e+09   \n",
       "8   000001.SZ   20190329  12.82  12.48  13.38  12.01  2.671303e+09   \n",
       "9   000001.SZ   20190228  12.36  11.20  12.66  10.90  1.911474e+09   \n",
       "10  000001.SZ   20190131  11.10   9.39  11.20   9.15  1.816315e+09   \n",
       "\n",
       "          amount  \n",
       "0   2.672014e+10  \n",
       "1   3.355394e+10  \n",
       "2   3.799487e+10  \n",
       "3   4.331356e+10  \n",
       "4   2.307291e+10  \n",
       "5   2.137764e+10  \n",
       "6   2.337029e+10  \n",
       "7   3.620546e+10  \n",
       "8   3.364583e+10  \n",
       "9   2.230669e+10  \n",
       "10  1.876003e+10  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tushare as ts\n",
    "pro = token_util.set_token()\n",
    "\n",
    "pro.monthly(ts_code='000001.SZ', start_date='20190101', end_date='20191201', fields='ts_code,trade_date,open,high,low,close,vol,amount')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 获取股东数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(stock_list)):\n",
    "    \n",
    "    try:\n",
    "        \n",
    "        stock_code = stock_list.iloc[i][\"ts_code\"]\n",
    "        data= data_util.pull_holder_num(stock_code, root_dir=root_dir)\n",
    "        \n",
    "    except:\n",
    "        time.sleep(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 获取上证指数信息"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "shang_zheng_data = get_price('000001.XSHG', start_date='2019-01-01', end_date='2019-11-11', frequency='daily', \n",
    "                              fields=None, skip_paused=False, fq='pre')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "date_index = []\n",
    "index_list = shang_zheng_data._stat_axis.values\n",
    "for index in index_list:\n",
    "    index = str(index)\n",
    "    date_index.append(index[:4] + index[5:7] + index[8:10])\n",
    "    \n",
    "shang_zheng_data[\"trade_date\"] = np.array(date_index)\n",
    "shang_zheng_data = shang_zheng_data.reset_index(drop=True)\n",
    "shang_zheng_data = data_util.cal_macd(shang_zheng_data, False)\n",
    "shang_zheng_data.to_csv(\"./data_pulled/000001.SH.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
